# -*- coding:utf-8 -*-
# @FileName  :meituan_t2.py
# @Time      :2021/7/31 11:30
import time
from save_mysql_mt import mysql_checking
import random
import re
from math import ceil
from multiprocessing import Pool
import requests
from fake_useragent import UserAgent
from time import sleep
import json
from lxml import html
import sys, io
from sql_ac.sql_1 import insert_all
from pypinyin import pinyin, lazy_pinyin, Style
from string import digits
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030')
sys.setrecursionlimit(1000000)
etree = html.etree
ua = [
    'Mozilla/5.0 (Linux; Android 8.1.0; JSN-AL00 Build/HONORJSN-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/045018 Mobile Safari/537.36 MMWEBID/3567 MicroMessenger/7.0.10.1580(0x27000AFE) Process/tools NetType/WIFI Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; Android 9; RMX1971 Build/PKQ1.190101.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.89 Mobile Safari/537.36 T7/11.19 SP-engine/2.15.0 baiduboxapp/11.19.5.10 (Baidu; P1 9)',
    'Mozilla/5.0 (Linux; Android 9; MIX 2 Build/PKQ1.190118.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/045111 Mobile Safari/537.36 MMWEBID/8679 MicroMessenger/7.0.10.1580(0x27000AFF) Process/tools NetType/WIFI Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; Android 10; LIO-AN00 Build/HUAWEILIO-AN00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.89 Mobile Safari/537.36 T7/11.19 SP-engine/2.15.0 baiduboxapp/11.19.5.10 (Baidu; P1 10) NABar/1.0',
    'Mozilla/5.0 (Linux; U; Android 9; zh-CN; RVL-AL09 Build/HUAWEIRVL-AL09) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.8.5.1065 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; U; Android 9; zh-CN; Redmi 6 Pro Build/PKQ1.180917.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.3.6.1016 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; U; Android 9; zh-CN; Redmi Note 8 Pro Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.8.5.1065 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; U; Android 9; zh-cn; MI 6X Build/PKQ1.180904.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.141 Mobile Safari/537.36 XiaoMi/MiuiBrowser/11.7.34',
    'Mozilla/5.0 (Linux; Android 9; JAT-AL00 Build/HONORJAT-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.89 Mobile Safari/537.36 T7/11.19 SP-engine/2.15.0 baiduboxapp/11.19.5.10 (Baidu; P1 9) NABar/1.0',
    'Mozilla/5.0 (Linux; Android 9; BLA-AL00 Build/HUAWEIBLA-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/045111 Mobile Safari/537.36 MMWEBID/9018 MicroMessenger/7.0.10.1580(0x27000AFF) Process/tools NetType/4G Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; Android 9; ONEPLUS A5010 Build/PKQ1.180716.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/045114 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; U; Android 9; zh-cn; MI 9 SE Build/PKQ1.181121.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.141 Mobile Safari/537.36 XiaoMi/MiuiBrowser/11.7.34',
    'Mozilla/5.0 (Linux; Android 10; TNY-AL00 Build/HUAWEITNY-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/45016 Mobile Safari/537.36 MMWEBID/2551 MicroMessenger/7.0.10.1580(0x27000AFE) Process/tools NetType/WIFI Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; U; Android 10; zh-cn; TNY-AL00 Build/HUAWEITNY-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/10.0 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; Android 9; JKM-AL00 Build/HUAWEIJKM-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.89 Mobile Safari/537.36 T7/11.19 SP-engine/2.15.0 baiduboxapp/11.19.5.10 (Baidu; P1 9)',
    'Mozilla/5.0 (Linux; Android 9; Lenovo L38111 Build/PKQ1.190127.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.89 Mobile Safari/537.36 T7/11.17 SP-engine/2.13.0 baiduboxapp/11.17.0.13 (Baidu; P1 9)',
    'Mozilla/5.0 (Linux; Android 9; vivo X21 Build/PKQ1.180819.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/76.0.3809.89 Mobile Safari/537.36 T7/11.19 SP-engine/2.15.0 baiduboxapp/11.19.5.10 (Baidu; P1 9)',
    'Mozilla/5.0 (Linux; Android 10; ELE-AL00 Build/HUAWEIELE-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/045104 Mobile Safari/537.36 MMWEBID/6317 MicroMessenger/7.0.10.1580(0x27000AFC) Process/tools NetType/WIFI Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; U; Android 10; zh-cn; HMA-TL00 Build/HUAWEIHMA-TL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/10.1 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; Android 9; Redmi Note 8 Pro Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.83 Mobile Safari/537.36 T7/11.16 lite baiduboxapp/4.11.5.11 (Baidu; P1 9)',
    'Mozilla/5.0 (Linux; U; Android 9; zh-cn; PCGM00 Build/PKQ1.190519.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/10.1 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; Android 9; JKM-AL00 Build/HUAWEIJKM-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/045118 Mobile Safari/537.36 MMWEBID/2787 MicroMessenger/7.0.10.1580(0x27000AFC) Process/tools NetType/WIFI Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; U; Android 10; zh-CN; ONEPLUS A6000 Build/QKQ1.190716.003) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.10.0.1163 UCTurbo/1.9.3.900 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; U; Android 10; zh-CN; VOG-AL10 Build/HUAWEIVOG-AL10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.8.5.1065 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; U; Android 9; zh-cn; SM-G9650 Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/10.0 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; Android 10; JSN-AL00 Build/HONORJSN-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 XWEB/1168 MMWEBSDK/191201 Mobile Safari/537.36 MMWEBID/6118 MicroMessenger/7.0.10.1580(0x27000AFE) Process/toolsmp NetType/4G Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; Android 10; VOG-TL00 Build/HUAWEIVOG-TL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/45016 Mobile Safari/537.36 MMWEBID/8723 MicroMessenger/7.0.10.1580(0x27000AFC) Process/tools NetType/4G Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; U; Android 9; zh-cn; JSN-AL00a Build/HONORJSN-AL00a) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/10.1 Mobile Safari/537.36',
    'Mozilla/5.0 (Linux; Android 10; ELE-AL00 Build/HUAWEIELE-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 XWEB/1168 MMWEBSDK/191201 Mobile Safari/537.36 MMWEBID/6553 MicroMessenger/7.0.10.1580(0x27000AFC) Process/toolsmp NetType/WIFI Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; Android 9; V1829A Build/PKQ1.181030.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/045105 Mobile Safari/537.36 MMWEBID/1118 MicroMessenger/7.0.10.1580(0x27000AFD) Process/tools NetType/4G Language/zh_CN ABI/arm64',
    'Mozilla/5.0 (Linux; Android 9; Note9 Build/PKQ1.181203.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.83 Mobile Safari/537.36 T7/11.15 baiduboxapp/11.15.5.10 (Baidu; P1 9)',
]
ua1 = UserAgent()
tunnel = ""
username = ""
password = ""
proxie = {
    "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
    "https": "https://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
}
kv = {
    'User-agent': ua1.random,
}

err = 0


def main(q, w):
    global err
    ct2 = q.strip().split(":")[1]
    ret1 = pinyin(ct2, heteronym=True, style=Style.TONE3)
    k = ''
    for i in ret1:
        k += i[0]
    res = k.translate(str.maketrans('', '', digits))
    b1 = {'长治': 'changzhi', '朝阳': 'chaoyang', '六安': 'liuan', '漯河': 'luohe', '铜仁': 'tongrendiqu', '毕节': 'bijiediqu', '巩义': 'gongyishi', '蓬莱': 'penglaiqu',
          '乐陵': 'laoling',
          '天长': 'tianchang', '长葛': 'changge', '宜都': 'yidu', '恒春': 'hengchun', '嘉义市': 'jiayi', '北碚': 'beipei', '邵东': 'shaodongshi', '乐亭': 'laoting',
          '滦县': 'luanzhoushi', '淮阳': 'huaiyangqu',
          '芜湖县': 'wuhu', '繁昌': 'fanchangqu', '广德': 'guangdeshi', '鲁山': 'lushanls', '太谷': 'taiguqu', '长垣': 'changyuanshi', '怀仁': 'huairenxian', '中牟': 'zhongmou',
          '单县': 'shanxian', '华亭': 'huatingshi', '浚县': 'xunxian',
          '洪洞': 'hongtong', '茌平': 'chipingqu', '彬县': 'binzhoushi', '京山': 'jingshanshi', '佛冈': 'fogang', '什邡': 'shifang', '于都': 'yudu', '珲春': 'hunchun',
          '平果': 'pingguoshi',
          '射洪': 'shehongshi', '潜山': 'qianshanshi', '任县': 'renzequ',
          '南和': 'nanhequ', '赤水': 'chishui', '泊头市': 'botou', '洛宁': 'luoningxian', '茶陵': 'chalingxian', '长丰县': 'changfengxian', '宁陵': 'ninglingxian',
          '南郑区': 'nanzhengxian',
          '酉阳土家族苗族自治县': 'youyangtujiazumiaozuzizh', '秀山土家族苗族自治县': 'xiushantujiazumiaozuzizh', '长清区': 'changqingqu', '嫩江县': 'nenjiangshi', '新津县': 'xinjinqu',
          '靖西市': 'jingxixian',
          '澄江县': 'chengjiangshi', '丰都': 'fengdu', '彭水苗族土家族自治县': 'pengshuizizhixian', '大厂回族自治县': 'dachangzizhixian', '大悟县': 'dawuxia', '景谷': 'jingguxian',
          '富民': 'fuminxian',
          '禄劝彝族苗族自治县': 'luquanxian', '石林彝族自治县': 'shilinxian'}
    if q.strip() in b:
        res = b1[f'{ct2}']
    key1 = w.strip().split(":")[1]
    try:
        r = requests.get(f'http://i.meituan.com/{res}/all/?cid={key1}&p=1&stid_b=3&cateType=poi', verify=False, headers=kv, proxies=proxie, timeout=30,
                         stream=True).text
        w1 = json.loads(re.findall('(id="filterData">)(.*)(</script>)', r)[0][1])['BizAreaList'][0]['count']
    except:
        sleep(10)
        print(f'http://i.meituan.com/{res}/all/?cid={key1}&p=1&stid_b=3&cateType=poi')
        if err < 5:
            err += 1
            return main(q, w)
        else:
            w1 = 1
            input('日你大灞')
    page = ceil(int(w1) / 50)
    err = 0
    if page > 67:
        page = 67
    for i in range(1, page):
        next(i, res, key1, w)


def next(page, q, w, key):
    try:
        r = requests.get(f'http://i.meituan.com/{q}/all/?cid={w}&p={page}&stid_b=3&cateType=poi', verify=False, headers=kv, timeout=30).text
    except:
        sleep(1)
        return next(page, q, w, key)
    data = etree.HTML(r)
    href = data.xpath("//dd[@class='poi-list-item']/a/@href")
    for j in href:
        spider(j, key, w)
    print(f'http://i.meituan.com/{q}/all/?cid={w}&p={page}&stid_b=3&cateType=poi')
    print(f'第{page}页,{q},{w},{key}')


def spider(w, key, ct1):
    sleep(1)
    kv1 = {
        'user-agent': random.choice(ua),
        'referer': 'https://meishi.meituan.com/i/?ci={}&stid_b=1&cevent=imt/homepage/category1/1'.format(ct1),
    }
    try:
        r = requests.get(w, verify=False, headers=kv1, proxies=proxie, timeout=30).text
    except Exception as err1:
        sleep(2)
        print(err1, '性情错误', w)
        if str(err1).find('No schema supplied. Perhaps you meant') != -1:
            return ''
        return spider(w, key, ct1)
    print(w)
    # print(r)
    try:
        jsdata = json.loads(re.findall(r'(window._appState = )(.*)(;</)', r)[0][1])
    except Exception as err:
        sleep(2)
        print(re.findall(r'(window._appState = )(.*)(;</)', r), '请求失败-。-', err)
        return spider(w, key, ct1)
    try:
        phone = jsdata['poiInfo']['phone']
        name = jsdata['poiInfo']['name']
        addr = jsdata['poiInfo']['addr']
    except Exception as wz:
        sleep(2)
        print(wz, '位置错误')
        return spider(w, key, ct1)
    # print(name +phone)
    pp = pe(jsdata['poiInfo']['phone'])
    if pp != '1':
        if mysql_checking(pp):
            try:
                insert_all('meituan_spider',
                           jsdata['poiInfo']['name'].replace("'", "").replace("'", ""), jsdata['poiInfo']['phone'],
                           pp, key.strip().split(":")[0].replace("'", "").replace("'", ""), '',
                           jsdata['poiInfo']['name'].replace("'", "").replace("'", ""), 2,
                           '0', 1, '', '', '', '', '', '', 2, '', '', addr.replace("'", "").replace("'", ""), '', '',
                           w, '', '', '', 0, '', 0, '', 0, 0, 0, 0, 0,
                           str(time.strftime('%Y-%m-%d %H:%M:%S')), str(time.strftime('%Y-%m-%d %H:%M:%S')), '', 0, 0, '', 0)
                print('入库')
            except Exception as s:
                print('入库错误了', s)


def pe(r):
    lu = r.split("/")
    for p in lu:
        if len(p) == 11 and p[0] == '1':
            return p
    return '1'


if __name__ == "__main__":
    with open('cityid11.txt') as fs:
        citys = fs.readlines()
    with open('fodid.txt') as fs:
        keys = fs.readlines()
    '''
    美食:1    0
    火锅:17   1
    聚餐宴请:395    2
    小吃快餐:36   3 -
    面包甜点:11    4
    饮品店:21329   5
    生日蛋糕:20097    6
    生鲜水果:20638    7
    烧烤:54       8
    '''
    b = ['山西:长治:0131', '辽宁:朝阳:0160', '安徽:六安:0203', '河南:漯河:0242', '贵州:铜仁:0325', '贵州:毕节:0327', '河南:巩义:0553', '山东:蓬莱:0587', '山东:乐陵:0611', '安徽:天长:0626',
         '河南:长葛:0627', '湖北:宜都:0656', '台湾:恒春:0712', '台湾:嘉义市:0719', '重庆:北碚:0731', '湖南:邵东:0736', '河北:乐亭:0744', '河北:滦县:0745', '河南:淮阳:0758', '安徽:芜湖县:0773',
         '安徽:繁昌:0774', '安徽:广德:0780', '河南:鲁山:0786', '山西:太谷:0790', '河南:长垣:0795', '山西:怀仁:0806', '河南:中牟:0821', '山东:单县:0830', '甘肃:华亭:0856', '河南:浚县:0863',
         '山西:洪洞:0867', '山东:茌平:0877', '陕西:彬县:0915', '湖北:京山:0918', '广东:佛冈:0921', '四川:什邡:0953', '江西:于都:0958', '吉林:珲春:0977', '广西:平果:0991', '四川:射洪:0992',
         '安徽:潜山:1004', '河北:任县:1012', '河北:南和:1013', '贵州:赤水:1018', '河北:泊头市:1074', '河南:洛宁:1087', '湖南:茶陵:1097', '安徽:长丰县:1100', '河南:宁陵:1112', '陕西:南郑区:1125',
         '重庆:酉阳土家族苗族自治县:1127', '重庆:秀山土家族苗族自治县:1129', '山东:长清区:1157', '黑龙江:嫩江县:1166', '四川:新津县:1167', '广西:靖西市:1197', '云南:澄江县:1205', '重庆:丰都:1231',
         '重庆:彭水苗族土家族自治县:1232', '河北:大厂回族自治县:1233', '湖北:大悟县:1243', '云南:景谷:1252', '云南:富民:1253', '云南:禄劝彝族苗族自治县:1254', '云南:石林彝族自治县:1272']
    for i in citys[130:150]:
        for j in keys[:8]:
            # input(i+j)
            print(i + j, '为曹操啊从')
            main(i, j)
# 011019207413
